#include "WebPage.h"
#include "simhash/Simhasher.hpp"
#include <iostream>
#include <algorithm>
#include <regex>
#include <codecvt>
#include <unistd.h>
using namespace simhash;

//使用正则表达式判断分词是否是有效的中文
bool isChinese(const string& word){
    //正则表达式匹配中文字符
    regex chinese_regex("^[\u4e00-\u9fa5]+$");
    return regex_match(word, chinese_regex);
}

//判断字符是否是英文字符或中文字符，不包括标点符号
bool isChineseOrEnglish(char32_t ch) {
    // 中文字符范围
    if (ch >= 0x4e00 && ch <= 0x9fa5) return true;
    // 英文字符范围
    if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')) return true;
    return false;
}

//去除字符串中非英文字符或中文字符的部分
string removeNonChineseOrEnglish(const string &input) {
    wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
    u32string u32input = conv.from_bytes(input);
    u32string u32output;
    
    for (char32_t ch : u32input) {
        if (isChineseOrEnglish(ch)) {
            u32output += ch;
        }
    }
    
    return conv.to_bytes(u32output);
}

//离线阶段WebPage的构造函数
WebPage::WebPage(string &doc, Configuration *config, SplitTool &splitTool)
: _doc(doc){
    size_t startPos = 0;
    size_t endPos = 0;
    
    startPos = doc.find("<docid>") + string("<docid>").size();
    endPos = doc.find("</docid>");
    _docid = stoi(_doc.substr(startPos, endPos - startPos));
    
    startPos = doc.find("<title>") + string("<title>").size();
    endPos = doc.find("</title>");
    _title = _doc.substr(startPos, endPos - startPos);
    
    startPos = doc.find("<url>") + string("<url>").size();
    endPos = doc.find("</url>");
    _url = _doc.substr(startPos, endPos - startPos);
    
    startPos = doc.find("<content>") + string("<content>").size();
    endPos = doc.find("</content>");
    _content = _doc.substr(startPos, endPos - startPos);
    
    /* _summary = */ 

    //对文档内容进行分词和去停用词操作，统计词频
    vector<string> words;
    words = splitTool.cut(removeNonChineseOrEnglish(_content));
    /* words = splitTool.cut(_content); */
    set<string> &stopWords = config->getStopWordList();
    //使用正则表达式很耗费时间，虽然离线阶段的时间消耗并不重要
    //但使用正则的代价过大，每次编译都要等待很长时间
    for(auto &i: words){
        /* if(stopWords.count(i) == 0 && isChinese(i)){ */
        if(stopWords.count(i) == 0){
            _wordsMap[i]++;
        }
    }

    //获取词频最高的前20个词
    vector<pair<string, int>> vec(_wordsMap.begin(), _wordsMap.end());
    //使用lambda表达式根据值对vector进行排序
    sort(vec.begin(), vec.end(), [](const pair<string, int>& a, const pair<string, int>& b){
        return a.second > b.second;
    });

    int num = 20 < vec.size() ? 20 : vec.size();
    for(int i = 0; i < num; i++){
        _topWords.push_back(vec[i].first);
    }
    
    //将词频最高的前20个词的集合添加到字符串的末尾
    string tops;
    for(auto &i: _topWords){
        tops += i;
        tops += " ";
    }
    _doc += "<tops>" + tops + "</tops>";
    
    /* cout << "doc: " << _doc << endl; */
    /* cout << "id: " << _docid << endl; */
    /* cout << "title: " << _title << endl; */
    /* cout << "url: " << _url << endl; */
    /* cout << "content: " << _content << endl; */
    /* for(auto &i: _wordsMap){ */
    /*     cout << i << " "; */
    /* } */
    /* cout << endl; */
    /* for(auto &i: _topWords){ */
    /*     cout << i << " "; */
    /* } */
    /* cout << endl; */

}

//在线阶段WebPage的构造函数
WebPage::WebPage(string &doc): _doc(doc){
    size_t startPos = 0;
    size_t endPos = 0;
    startPos = doc.find("<docid>") + string("<docid>").size();
    endPos = doc.find("</docid>");
    _docid = stoi(_doc.substr(startPos, endPos - startPos));
    
    startPos = doc.find("<title>") + string("<title>").size();
    endPos = doc.find("</title>");
    _title = _doc.substr(startPos, endPos - startPos);
    
    startPos = doc.find("<url>") + string("<url>").size();
    endPos = doc.find("</url>");
    _url = _doc.substr(startPos, endPos - startPos);
    
    startPos = doc.find("<content>") + string("<content>").size();
    endPos = doc.find("</content>");
    _content = _doc.substr(startPos, endPos - startPos);
    
    startPos = doc.find("<tops>") + string("<tops>").size();
    endPos = doc.find("</tops>");
    
    //下面这一步不是必要的，因为可以直接把tops作为字符串发送出去
    string tops = doc.substr(startPos, endPos - startPos);
    istringstream iss(tops);
    string word;
    while(iss >> word){
        _topWords.push_back(word);
    }
}

WebPage::WebPage(){

}

WebPage::~WebPage(){

}

int WebPage::getDocId(){
    return _docid;
}

string WebPage::getDoc(){
    return _doc;
}

string WebPage::getDocContent(){
    return _content;
}

map<string, int> & WebPage::getWordsMap(){
    return _wordsMap;
}

void WebPage::getTopK(vector<string> &topKList, int k){
    vector<pair<string, int>> wordCounts(_wordsMap.begin(), _wordsMap.end());
    sort(wordCounts.begin(), wordCounts.end(), [](const pair<string, int>& a, const pair<string, int>& b) {
        return a.second > b.second; //按照count从大到小排序
    });
    for (int i = 0; i < min(k, (int)wordCounts.size()); ++i) {
        topKList.push_back(wordCounts[i].first);
    }
}


